package com.jinghang.WordCountApplication

import com.jinghang.spark_base._010_RDD._010CreateRDD.{conf, sparkContext}
import org.apache.spark.{SparkConf, SparkContext}

object WordCountLocal {

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
      .setAppName("appName")//设置一个Application的id,用于在ui上辨别该Application
      .setMaster("local[2]")//local带表本地执行，【n】代表使用几个core,使用n个线程执行任务
      .set("spark.testing.memory", "471859200")
     val sparkContext = new SparkContext(conf)

    val rdd = sparkContext.textFile("hdfs://192.168.43.54:8020/input/wcFile")
    val flatMapRdd = rdd.flatMap(line => line.split(" "))
    val mapRdd = flatMapRdd.map(x => (x, 1))
    val reduceByKeyRdd = mapRdd.reduceByKey((a, b) => (a + b)).sortByKey()

    reduceByKeyRdd.saveAsTextFile("data/output/wcResult")

    //reduceByKeyRdd.collect().foreach(println)

  }




}
